<!DOCTYPE html>
<html style="display: none;" lang="zh">
    <head><meta name="generator" content="Hexo 3.9.0">
    <!-- hexo-inject:begin --><!-- hexo-inject:end --><meta charset="utf-8">
    <!--
        © Material Theme
        https://github.com/viosey/hexo-theme-material
        Version: 1.5.2 -->
    <script>
        window.materialVersion = "1.5.2"
        // Delete localstorage with these tags
        window.oldVersion = [
            'codestartv1',
            '1.3.4',
            '1.4.0',
            '1.4.0b1',
            '1.5.0'
        ]
    </script>

    <!-- dns prefetch -->
    <meta http-equiv="x-dns-prefetch-control" content="on">














    <!-- Meta & Info -->
    <meta http-equiv="X-UA-Compatible" content="IE=Edge,chrome=1">
    <meta name="renderer" content="webkit">
    <meta name="viewport" content="width=device-width, initial-scale=1, user-scalable=no">

    <!-- Title -->
    
    <title>
        
            HDFS | 
        
        fengweizan&#39;s BLOG
    </title>

    <!-- Favicons -->
    <link rel="icon shortcut" type="image/ico" href="/img/favicon.png">
    <link rel="icon" href="/img/favicon.png">

    <meta name="format-detection" content="telephone=no">
    <meta name="description" itemprop="description" content="my blog with material design">
    <meta name="keywords" content=",hadoop">
    <meta name="theme-color" content="#0097A7">

    <!-- Disable Fucking Bloody Baidu Tranformation -->
    <meta http-equiv="Cache-Control" content="no-transform">
    <meta http-equiv="Cache-Control" content="no-siteapp">

    <!--[if lte IE 9]>
        <link rel="stylesheet" href="/css/ie-blocker.css">

        
            <script src="/js/ie-blocker.zhCN.js"></script>
        
    <![endif]-->

    <!-- Import lsloader -->
    <script>(function(){window.lsloader={jsRunSequence:[],jsnamemap:{},cssnamemap:{}};lsloader.removeLS=function(a){try{localStorage.removeItem(a)}catch(b){}};lsloader.setLS=function(a,c){try{localStorage.setItem(a,c)}catch(b){}};lsloader.getLS=function(a){var c="";try{c=localStorage.getItem(a)}catch(b){c=""}return c};versionString="/*"+(window.materialVersion||"unknownVersion")+"*/";lsloader.clean=function(){try{var b=[];for(var a=0;a<localStorage.length;a++){b.push(localStorage.key(a))}b.forEach(function(e){var f=lsloader.getLS(e);if(window.oldVersion){var d=window.oldVersion.reduce(function(g,h){return g||f.indexOf("/*"+h+"*/")!==-1},false);if(d){lsloader.removeLS(e)}}})}catch(c){}};lsloader.clean();lsloader.load=function(f,a,b,d){if(typeof b==="boolean"){d=b;b=undefined}d=d||false;b=b||function(){};var e;e=this.getLS(f);if(e&&e.indexOf(versionString)===-1){this.removeLS(f);this.requestResource(f,a,b,d);return}if(e){var c=e.split(versionString)[0];if(c!=a){console.log("reload:"+a);this.removeLS(f);this.requestResource(f,a,b,d);return}e=e.split(versionString)[1];if(d){this.jsRunSequence.push({name:f,code:e});this.runjs(a,f,e)}else{document.getElementById(f).appendChild(document.createTextNode(e));b()}}else{this.requestResource(f,a,b,d)}};lsloader.requestResource=function(b,e,a,c){var d=this;if(c){this.iojs(e,b,function(h,f,g){d.setLS(f,h+versionString+g);d.runjs(h,f,g)})}else{this.iocss(e,b,function(f){document.getElementById(b).appendChild(document.createTextNode(f));d.setLS(b,e+versionString+f)},a)}};lsloader.iojs=function(d,b,g){var a=this;a.jsRunSequence.push({name:b,code:""});try{var f=new XMLHttpRequest();f.open("get",d,true);f.onreadystatechange=function(){if(f.readyState==4){if((f.status>=200&&f.status<300)||f.status==304){if(f.response!=""){g(d,b,f.response);return}}a.jsfallback(d,b)}};f.send(null)}catch(c){a.jsfallback(d,b)}};lsloader.iocss=function(f,c,h,a){var b=this;try{var g=new XMLHttpRequest();g.open("get",f,true);g.onreadystatechange=function(){if(g.readyState==4){if((g.status>=200&&g.status<300)||g.status==304){if(g.response!=""){h(g.response);a();return}}b.cssfallback(f,c,a)}};g.send(null)}catch(d){b.cssfallback(f,c,a)}};lsloader.iofonts=function(f,c,h,a){var b=this;try{var g=new XMLHttpRequest();g.open("get",f,true);g.onreadystatechange=function(){if(g.readyState==4){if((g.status>=200&&g.status<300)||g.status==304){if(g.response!=""){h(g.response);a();return}}b.cssfallback(f,c,a)}};g.send(null)}catch(d){b.cssfallback(f,c,a)}};lsloader.runjs=function(f,c,e){if(!!c&&!!e){for(var b in this.jsRunSequence){if(this.jsRunSequence[b].name==c){this.jsRunSequence[b].code=e}}}if(!!this.jsRunSequence[0]&&!!this.jsRunSequence[0].code&&this.jsRunSequence[0].status!="failed"){var a=document.createElement("script");a.appendChild(document.createTextNode(this.jsRunSequence[0].code));a.type="text/javascript";document.getElementsByTagName("head")[0].appendChild(a);this.jsRunSequence.shift();if(this.jsRunSequence.length>0){this.runjs()}}else{if(!!this.jsRunSequence[0]&&this.jsRunSequence[0].status=="failed"){var d=this;var a=document.createElement("script");a.src=this.jsRunSequence[0].path;a.type="text/javascript";this.jsRunSequence[0].status="loading";a.onload=function(){d.jsRunSequence.shift();if(d.jsRunSequence.length>0){d.runjs()}};document.body.appendChild(a)}}};lsloader.tagLoad=function(b,a){this.jsRunSequence.push({name:a,code:"",path:b,status:"failed"});this.runjs()};lsloader.jsfallback=function(c,b){if(!!this.jsnamemap[b]){return}else{this.jsnamemap[b]=b}for(var a in this.jsRunSequence){if(this.jsRunSequence[a].name==b){this.jsRunSequence[a].code="";this.jsRunSequence[a].status="failed";this.jsRunSequence[a].path=c}}this.runjs()};lsloader.cssfallback=function(e,c,b){if(!!this.cssnamemap[c]){return}else{this.cssnamemap[c]=1}var d=document.createElement("link");d.type="text/css";d.href=e;d.rel="stylesheet";d.onload=d.onerror=b;var a=document.getElementsByTagName("script")[0];a.parentNode.insertBefore(d,a)};lsloader.runInlineScript=function(c,b){var a=document.getElementById(b).innerText;this.jsRunSequence.push({name:c,code:a});this.runjs()}})();</script>

    <!-- Import queue -->
    <script>function Queue(){this.dataStore=[];this.offer=b;this.poll=d;this.execNext=a;this.debug=false;this.startDebug=c;function b(e){if(this.debug){console.log("Offered a Queued Function.")}if(typeof e==="function"){this.dataStore.push(e)}else{console.log("You must offer a function.")}}function d(){if(this.debug){console.log("Polled a Queued Function.")}return this.dataStore.shift()}function a(){var e=this.poll();if(e!==undefined){if(this.debug){console.log("Run a Queued Function.")}e()}}function c(){this.debug=true}}var queue=new Queue();</script>

    <!-- Import CSS -->
    
        <style id="material_css"></style><script>if(typeof window.lsLoadCSSMaxNums === "undefined")window.lsLoadCSSMaxNums = 0;window.lsLoadCSSMaxNums++;lsloader.load("material_css","/css/material.min.css?Z7a72R1E4SxzBKR/WGctOA==",function(){if(typeof window.lsLoadCSSNums === "undefined")window.lsLoadCSSNums = 0;window.lsLoadCSSNums++;if(window.lsLoadCSSNums == window.lsLoadCSSMaxNums)document.documentElement.style.display="";}, false)</script>
        <style id="style_css"></style><script>if(typeof window.lsLoadCSSMaxNums === "undefined")window.lsLoadCSSMaxNums = 0;window.lsLoadCSSMaxNums++;lsloader.load("style_css","/css/style.min.css?MKetZV3cUTfDxvMffaOezg==",function(){if(typeof window.lsLoadCSSNums === "undefined")window.lsLoadCSSNums = 0;window.lsLoadCSSNums++;if(window.lsLoadCSSNums == window.lsLoadCSSMaxNums)document.documentElement.style.display="";}, false)</script>

        
            
                <style id="prettify_css"></style><script>if(typeof window.lsLoadCSSMaxNums === "undefined")window.lsLoadCSSMaxNums = 0;window.lsLoadCSSMaxNums++;lsloader.load("prettify_css","/css/prettify.min.css?zp8STOU9v89XWFEnN+6YmQ==",function(){if(typeof window.lsLoadCSSNums === "undefined")window.lsLoadCSSNums = 0;window.lsLoadCSSNums++;if(window.lsLoadCSSNums == window.lsLoadCSSMaxNums)document.documentElement.style.display="";}, false)</script>
                <style id="prettify_theme"></style><script>if(typeof window.lsLoadCSSMaxNums === "undefined")window.lsLoadCSSMaxNums = 0;window.lsLoadCSSMaxNums++;lsloader.load("prettify_theme","/css/prettify/github-v2.min.css?AfzKxt++K+/lhZBlSjnxwg==",function(){if(typeof window.lsLoadCSSNums === "undefined")window.lsLoadCSSNums = 0;window.lsLoadCSSNums++;if(window.lsLoadCSSNums == window.lsLoadCSSMaxNums)document.documentElement.style.display="";}, false)</script>
            
        

    

    

    <!-- Config CSS -->

<!-- Other Styles -->
<style>
  body, html {
    font-family: Roboto, "Helvetica Neue", Helvetica, "PingFang SC", "Hiragino Sans GB", "Microsoft YaHei", "微软雅黑", Arial, sans-serif;
    overflow-x: hidden !important;
  }
  
  code {
    font-family: Consolas, Monaco, 'Andale Mono', 'Ubuntu Mono', monospace;
  }

  a {
    color: #00838F;
  }

  .mdl-card__media,
  #search-label,
  #search-form-label:after,
  #scheme-Paradox .hot_tags-count,
  #scheme-Paradox .sidebar_archives-count,
  #scheme-Paradox .sidebar-colored .sidebar-header,
  #scheme-Paradox .sidebar-colored .sidebar-badge{
    background-color: #0097A7 !important;
  }

  /* Sidebar User Drop Down Menu Text Color */
  #scheme-Paradox .sidebar-colored .sidebar-nav>.dropdown>.dropdown-menu>li>a:hover,
  #scheme-Paradox .sidebar-colored .sidebar-nav>.dropdown>.dropdown-menu>li>a:focus {
    color: #0097A7 !important;
  }

  #post_entry-right-info,
  .sidebar-colored .sidebar-nav li:hover > a,
  .sidebar-colored .sidebar-nav li:hover > a i,
  .sidebar-colored .sidebar-nav li > a:hover,
  .sidebar-colored .sidebar-nav li > a:hover i,
  .sidebar-colored .sidebar-nav li > a:focus i,
  .sidebar-colored .sidebar-nav > .open > a,
  .sidebar-colored .sidebar-nav > .open > a:hover,
  .sidebar-colored .sidebar-nav > .open > a:focus,
  #ds-reset #ds-ctx .ds-ctx-entry .ds-ctx-head a {
    color: #0097A7 !important;
  }

  .toTop {
    background: #757575 !important;
  }

  .material-layout .material-post>.material-nav,
  .material-layout .material-index>.material-nav,
  .material-nav a {
    color: #757575;
  }

  #scheme-Paradox .MD-burger-layer {
    background-color: #757575;
  }

  #scheme-Paradox #post-toc-trigger-btn {
    color: #757575;
  }

  .post-toc a:hover {
    color: #00838F;
    text-decoration: underline;
  }

</style>


<!-- Theme Background Related-->

    <style>
      body {
        background-color: #F5F5F5;
      }

      /* blog_info bottom background */
      #scheme-Paradox .material-layout .something-else .mdl-card__supporting-text {
        background-color: #fff;
      }
    </style>




<!-- Fade Effect -->

    <style>
      .fade {
        transition: all 800ms linear;
        -webkit-transform: translate3d(0,0,0);
        -moz-transform: translate3d(0,0,0);
        -ms-transform: translate3d(0,0,0);
        -o-transform: translate3d(0,0,0);
        transform: translate3d(0,0,0);
        opacity: 1;
      }

      .fade.out{
        opacity: 0;
      }
    </style>


<!-- Import Font -->
<!-- Import Roboto -->

    <link href="https://fonts.googleapis.com/css?family=Roboto:300,400,500" rel="stylesheet">


<!-- Import Material Icons -->


    <style id="material_icons"></style><script>if(typeof window.lsLoadCSSMaxNums === "undefined")window.lsLoadCSSMaxNums = 0;window.lsLoadCSSMaxNums++;lsloader.load("material_icons","/css/material-icons.css?pqhB/Rd/ab0H2+kZp0RDmw==",function(){if(typeof window.lsLoadCSSNums === "undefined")window.lsLoadCSSNums = 0;window.lsLoadCSSNums++;if(window.lsLoadCSSNums == window.lsLoadCSSMaxNums)document.documentElement.style.display="";}, false)</script>




    <!-- Import jQuery -->
    
        <script>lsloader.load("jq_js","/js/jquery.min.js?qcusAULNeBksqffqUM2+Ig==", true)</script>
    

    <!-- WebAPP Icons -->
    <meta name="mobile-web-app-capable" content="yes">
    <meta name="application-name" content="fengweizan&#39;s BLOG">
    <meta name="msapplication-starturl" content="https://fclehly.github.io/2019/08/06/HDFS/">
    <meta name="msapplication-navbutton-color" content="#0097A7">
    <meta name="apple-mobile-web-app-capable" content="yes">
    <meta name="apple-mobile-web-app-title" content="fengweizan&#39;s BLOG">
    <meta name="apple-mobile-web-app-status-bar-style" content="black">
    <link rel="apple-touch-icon" href="/img/favicon.png">

    <!-- Site Verification -->
    
    

    <!-- RSS -->
    

    <!-- The Open Graph protocol -->
    <meta property="og:url" content="https://fclehly.github.io/2019/08/06/HDFS/">
    <meta property="og:type" content="blog">
    <meta property="og:title" content="HDFS | fengweizan&#39;s BLOG">
    <meta property="og:image" content="/img/favicon.png">
    <meta property="og:description" content="my blog with material design">
    <meta property="og:article:tag" content="hadoop"> 

    
        <meta property="article:published_time" content="Tue Aug 06 2019 22:18:47 GMT+0800">
        <meta property="article:modified_time" content="Wed Aug 14 2019 23:31:30 GMT+0800">
    

    <!-- The Twitter Card protocol -->
    <meta name="twitter:card" content="summary_large_image">

    <!-- Add canonical link for SEO -->
    
        <link rel="canonical" href="https://fclehly.github.io/2019/08/06/HDFS/index.html">
    

    <!-- Structured-data for SEO -->
    
        


<script type="application/ld+json">
{
    "@context": "https://schema.org",
    "@type": "BlogPosting",
    "mainEntityOfPage": "https://fclehly.github.io/2019/08/06/HDFS/index.html",
    "headline": "HDFS",
    "datePublished": "Tue Aug 06 2019 22:18:47 GMT+0800",
    "dateModified": "Wed Aug 14 2019 23:31:30 GMT+0800",
    "author": {
        "@type": "Person",
        "name": "FWZ",
        "image": {
            "@type": "ImageObject",
            "url": "/img/avatar.png"
        },
        "description": "Hi, nice to meet you"
    },
    "publisher": {
        "@type": "Organization",
        "name": "fengweizan&#39;s BLOG",
        "logo": {
            "@type":"ImageObject",
            "url": "/img/favicon.png"
        }
    },
    "keywords": ",hadoop",
    "description": "my blog with material design",
}
</script>


    

    <!-- Analytics -->
    
    
    

    <!-- Custom Head --><!-- hexo-inject:begin --><!-- hexo-inject:end -->
    

</head>


    
        <body id="scheme-Paradox" class="lazy">
            <!-- hexo-inject:begin --><!-- hexo-inject:end --><div class="material-layout  mdl-js-layout has-drawer is-upgraded">
                

                <!-- Main Container -->
                <main class="material-layout__content" id="main">

                    <!-- Top Anchor -->
                    <div id="top"></div>

                    
                        <!-- Hamburger Button -->
                        <button class="MD-burger-icon sidebar-toggle">
                            <span class="MD-burger-layer"></span>
                        </button>
                    

                    <!-- Post TOC -->

    
    <!-- Back Button -->
    <!--
    <div class="material-back" id="backhome-div" tabindex="0">
        <a class="mdl-button mdl-js-button mdl-js-ripple-effect mdl-button--icon"
           href="#" onclick="window.history.back();return false;"
           target="_self"
           role="button"
           data-upgraded=",MaterialButton,MaterialRipple">
            <i class="material-icons" role="presentation">arrow_back</i>
            <span class="mdl-button__ripple-container">
                <span class="mdl-ripple"></span>
            </span>
        </a>
    </div>
    -->


    <!-- Left aligned menu below button -->
    
    
    <button id="post-toc-trigger-btn"
        class="mdl-button mdl-js-button mdl-button--icon">
        <i class="material-icons">format_list_numbered</i>
    </button>

    <ul class="post-toc-wrap mdl-menu mdl-menu--bottom-left mdl-js-menu mdl-js-ripple-effect" for="post-toc-trigger-btn" style="max-height:80vh; overflow-y:scroll;">
        <ol class="post-toc"><li class="post-toc-item post-toc-level-1"><a class="post-toc-link" href="#HDFS"><span class="post-toc-number">1.</span> <span class="post-toc-text">HDFS</span></a><ol class="post-toc-child"><li class="post-toc-item post-toc-level-2"><a class="post-toc-link" href="#存储模型"><span class="post-toc-number">1.1.</span> <span class="post-toc-text">存储模型</span></a></li><li class="post-toc-item post-toc-level-2"><a class="post-toc-link" href="#架构模型"><span class="post-toc-number">1.2.</span> <span class="post-toc-text">架构模型</span></a><ol class="post-toc-child"><li class="post-toc-item post-toc-level-3"><a class="post-toc-link" href="#NameNode"><span class="post-toc-number">1.2.1.</span> <span class="post-toc-text">NameNode</span></a></li><li class="post-toc-item post-toc-level-3"><a class="post-toc-link" href="#NameNode持久化"><span class="post-toc-number">1.2.2.</span> <span class="post-toc-text">NameNode持久化</span></a></li><li class="post-toc-item post-toc-level-3"><a class="post-toc-link" href="#DataNode"><span class="post-toc-number">1.2.3.</span> <span class="post-toc-text">DataNode</span></a></li><li class="post-toc-item post-toc-level-3"><a class="post-toc-link" href="#HDFS优点"><span class="post-toc-number">1.2.4.</span> <span class="post-toc-text">HDFS优点</span></a></li><li class="post-toc-item post-toc-level-3"><a class="post-toc-link" href="#HDFS缺点"><span class="post-toc-number">1.2.5.</span> <span class="post-toc-text">HDFS缺点</span></a></li><li class="post-toc-item post-toc-level-3"><a class="post-toc-link" href="#SecondaryNameNode"><span class="post-toc-number">1.2.6.</span> <span class="post-toc-text">SecondaryNameNode</span></a></li><li class="post-toc-item post-toc-level-3"><a class="post-toc-link" href="#安全模式"><span class="post-toc-number">1.2.7.</span> <span class="post-toc-text">安全模式</span></a></li><li class="post-toc-item post-toc-level-3"><a class="post-toc-link" href="#Block的副本防置策略"><span class="post-toc-number">1.2.8.</span> <span class="post-toc-text">Block的副本防置策略</span></a></li><li class="post-toc-item post-toc-level-3"><a class="post-toc-link" href="#HDFS写流程"><span class="post-toc-number">1.2.9.</span> <span class="post-toc-text">HDFS写流程</span></a></li><li class="post-toc-item post-toc-level-3"><a class="post-toc-link" href="#HDFS读流程"><span class="post-toc-number">1.2.10.</span> <span class="post-toc-text">HDFS读流程</span></a></li><li class="post-toc-item post-toc-level-3"><a class="post-toc-link" href="#HDFS文件权限-POSIX"><span class="post-toc-number">1.2.11.</span> <span class="post-toc-text">HDFS文件权限 POSIX</span></a></li></ol></li><li class="post-toc-item post-toc-level-2"><a class="post-toc-link" href="#Hadoop2"><span class="post-toc-number">1.3.</span> <span class="post-toc-text">Hadoop2</span></a><ol class="post-toc-child"><li class="post-toc-item post-toc-level-3"><a class="post-toc-link" href="#Hadoop2产生背景"><span class="post-toc-number">1.3.1.</span> <span class="post-toc-text">Hadoop2产生背景</span></a></li><li class="post-toc-item post-toc-level-3"><a class="post-toc-link" href="#HDFS2"><span class="post-toc-number">1.3.2.</span> <span class="post-toc-text">HDFS2</span></a></li></ol></li></ol></li></ol>
    </ul>
    




<!-- Layouts -->

    <!-- Post Module -->
    <div class="material-post_container">

        <div class="material-post mdl-grid">
            <div class="mdl-card mdl-shadow--4dp mdl-cell mdl-cell--12-col">

                <!-- Post Header(Thumbnail & Title) -->
                
    <!-- Paradox Post Header -->
    
        
            <!-- Random Thumbnail -->
            <div class="post_thumbnail-random mdl-card__media mdl-color-text--grey-50">
            <script type="text/ls-javascript" id="post-thumbnail-script">
    var randomNum = Math.floor(Math.random() * 19 + 1);

    $('.post_thumbnail-random').attr('data-original', '/img/random/material-' + randomNum + '.png');
    $('.post_thumbnail-random').addClass('lazy');
</script>

        
    
            <p class="article-headline-p">
                HDFS
            </p>
        </div>





                
                    <!-- Paradox Post Info -->
                    <div class="mdl-color-text--grey-700 mdl-card__supporting-text meta">

    <!-- Author Avatar -->
    <div id="author-avatar">
        <img src="/img/avatar.png" width="44px" height="44px" alt="Author Avatar"/>
    </div>
    <!-- Author Name & Date -->
    <div>
        <strong>FWZ</strong>
        <span>8月 06, 2019</span>
    </div>

    <div class="section-spacer"></div>

    <!-- Favorite -->
    <!--
        <button id="article-functions-like-button" class="mdl-button mdl-js-button mdl-js-ripple-effect mdl-button--icon btn-like">
            <i class="material-icons" role="presentation">favorite</i>
            <span class="visuallyhidden">favorites</span>
        </button>
    -->

    <!-- Qrcode -->
    

    <!-- Tags (bookmark) -->
    
    <button id="article-functions-viewtags-button" class="mdl-button mdl-js-button mdl-js-ripple-effect mdl-button--icon">
        <i class="material-icons" role="presentation">bookmark</i>
        <span class="visuallyhidden">bookmark</span>
    </button>
    <ul class="mdl-menu mdl-menu--bottom-right mdl-js-menu mdl-js-ripple-effect" for="article-functions-viewtags-button">
        <li class="mdl-menu__item">
        <a class="post_tag-link" href="/tags/hadoop/">hadoop</a>
    </ul>
    

    <!-- Share -->
    
        <button id="article-fuctions-share-button" class="mdl-button mdl-js-button mdl-js-ripple-effect mdl-button--icon">
    <i class="material-icons" role="presentation">share</i>
    <span class="visuallyhidden">share</span>
</button>
<ul class="mdl-menu mdl-menu--bottom-right mdl-js-menu mdl-js-ripple-effect" for="article-fuctions-share-button">
    

    

    <!-- Share Weibo -->
    
        <a class="post_share-link" href="http://service.weibo.com/share/share.php?appkey=&title=HDFS&url=https://fclehly.github.io/2019/08/06/HDFS/index.html&pic=https://fclehly.github.io/img/favicon.png&searchPic=false&style=simple" target="_blank">
            <li class="mdl-menu__item">
                分享到微博
            </li>
        </a>
    

    <!-- Share Twitter -->
    
        <a class="post_share-link" href="https://twitter.com/intent/tweet?text=HDFS&url=https://fclehly.github.io/2019/08/06/HDFS/index.html&via=FWZ" target="_blank">
            <li class="mdl-menu__item">
                分享到 Twitter
            </li>
        </a>
    

    <!-- Share Facebook -->
    
        <a class="post_share-link" href="https://www.facebook.com/sharer/sharer.php?u=https://fclehly.github.io/2019/08/06/HDFS/index.html" target="_blank">
            <li class="mdl-menu__item">
                分享到 Facebook
            </li>
        </a>
    

    <!-- Share Google+ -->
    
        <a class="post_share-link" href="https://plus.google.com/share?url=https://fclehly.github.io/2019/08/06/HDFS/index.html" target="_blank">
            <li class="mdl-menu__item">
                分享到 Google+
            </li>
        </a>
    

    <!-- Share LinkedIn -->
    

    <!-- Share QQ -->
    

    <!-- Share Telegram -->
    
</ul>

    
</div>

                

                <!-- Post Content -->
                <div id="post-content" class="mdl-color-text--grey-700 mdl-card__supporting-text fade out">
    
        <h1 id="HDFS"><a href="#HDFS" class="headerlink" title="HDFS"></a>HDFS</h1><h2 id="存储模型"><a href="#存储模型" class="headerlink" title="存储模型"></a>存储模型</h2><p>文件线性切割成块（block），每个block可以根据其字节内容在原文件中的偏移量offset确定，即第一个block的第一个字节的offset为0、第二个block的第一个字节的offset为block_size。block分散存储到集群节点当中，单一文件的block大小一致，不同文件之间的block大小可以不一样。block可以设置副本数，block的副本分散在不同节点中（即副本不会出现在同一节点中），这样可以很好地支持多个程序处理文件，但副本数不能超过节点数量。文件上传可以设置block的大小和副本数，且已上传的文件block副本数可以调整，但是大小不能改变。只支持一次写入多次读取，同一时刻只有一个写入者。可以append追加数据，但不可以修改block。</p>
<h2 id="架构模型"><a href="#架构模型" class="headerlink" title="架构模型"></a>架构模型</h2><p>文件组成分为：</p>
<ul>
<li>元数据metadata，</li>
<li>文件数据</li>
</ul>
<p>主从架构：</p>
<ul>
<li>NameNode节点保存文件元数据：单节点，维护一个虚拟目录树，映射到DataNode的文件系统的Block。</li>
<li>DataNode节点保存文件block数据：多节点；</li>
</ul>
<p><img src="http://hadoop.apache.org/docs/stable/hadoop-project-dist/hadoop-hdfs/images/hdfsarchitecture.png" alt="hdfs-architecture"></p>
<p>DataNode和NameNode保持心跳，提交block列表；</p>
<p>HdfsClient和NameNode交互元数据信息；</p>
<p>HdfsClient和DataNode交互文件Block信息。</p>
<p>HdfsClient与NameNode交互后，向DataNode交互，NameNode起到了一定的负载均衡作用。</p>
<h3 id="NameNode"><a href="#NameNode" class="headerlink" title="NameNode"></a>NameNode</h3><p>基于内存存储，不会和磁盘发生交换。数据只存在内存中，但会通过快照或增量日志做持久化策略。</p>
<p>主要功能：</p>
<ul>
<li>接受客户端的读写服务；</li>
<li>收集DataNode汇报的Block列表信息。</li>
</ul>
<p>其保存的metadata信息包括：</p>
<ul>
<li>文件ownership， permissions；</li>
<li>文件大小，时间</li>
<li>block列表，block偏移量，位置信息；</li>
<li>block每个副本的位置（由DataNode上报）</li>
</ul>
<h3 id="NameNode持久化"><a href="#NameNode持久化" class="headerlink" title="NameNode持久化"></a>NameNode持久化</h3><p>NameNode的metadata在启动后会加载到内存；</p>
<p>metadata存储到磁盘的文件名为fsimage；</p>
<p>block的位置信息不会保存到fsimage；</p>
<p>edits记录对metadata的操作记录；</p>
<h3 id="DataNode"><a href="#DataNode" class="headerlink" title="DataNode"></a>DataNode</h3><p>本地磁盘目录存储数据block，使文件形式，同时存储block的元数据信息文件幂md5；</p>
<p>启动datanode时会向namenode汇报block信息；</p>
<p>通过向nanenode发送心跳保持与其联系（3秒1次），如果namenode在10分钟内没有收到datanode的信息，则认为其已经lost，并copy其上的block（从其他节点的副本拷贝扩散，保证副本数一致）到其他datanode。</p>
<h3 id="HDFS优点"><a href="#HDFS优点" class="headerlink" title="HDFS优点"></a>HDFS优点</h3><ul>
<li>高容错性：<ul>
<li>数据自动保存多个副本；</li>
<li>副本数据丢失，则会自动回复；</li>
</ul>
</li>
<li>适合批处理<ul>
<li>移动计算程序而不是数据；</li>
<li>数据位置暴露给计算框架；</li>
</ul>
</li>
<li>适合大数据处理<ul>
<li>GB、TB、PB</li>
<li>百万规模以上的文件数量</li>
<li>10k+节点</li>
</ul>
</li>
<li>可构建在廉价机器上（可靠性越高的机器越贵）<ul>
<li>通过多副本提高可靠性；</li>
<li>提供了容错和恢复机制；</li>
</ul>
</li>
</ul>
<h3 id="HDFS缺点"><a href="#HDFS缺点" class="headerlink" title="HDFS缺点"></a>HDFS缺点</h3><ul>
<li>低延迟数据访问<ul>
<li>毫秒级随机访问</li>
<li>低延迟与高吞吐率</li>
</ul>
</li>
<li>小文件存取<ul>
<li>占用NameNode大量内存</li>
<li>寻道时间超过读取时间</li>
</ul>
</li>
<li>并发写入、文件随机修改<ul>
<li>一个文件只能有一个写者</li>
<li>仅支持append</li>
</ul>
</li>
</ul>
<h3 id="SecondaryNameNode"><a href="#SecondaryNameNode" class="headerlink" title="SecondaryNameNode"></a>SecondaryNameNode</h3><p>SNN不是NN的备份（但是可以做备份），其主要工作是帮助NN合并edits log，减少NN的启动时间。<br>SNN执行合并时间：</p>
<ul>
<li>根据配置文件设置的时间间隔<code>fs.checkpoint.period</code>，默认为3600s；</li>
<li>根据配置文件设置edits log大小<code>fs.checkpoint.size</code>规定edits log文件的最大值默认是64MB</li>
</ul>
<h3 id="安全模式"><a href="#安全模式" class="headerlink" title="安全模式"></a>安全模式</h3><ul>
<li>namenode启动时，首先将映像文件fsimage载入内存，并执行编辑日志（edits）中的各项操作。</li>
<li>一旦在内存中成功建立文件系统元数据的映射，则创建一个新的fsimage文件（这个操作不需要SecondaryNameNode）和一个空的编辑日志，这里也是namenode唯一一次创建fsimage。</li>
<li>此刻namenode运行在安全模式。即namenode的文件系统对于客户端来说只是只读的。（显示目录、文件内容等。写、删除、重命名等操作都会失败）</li>
<li>在此阶段Namenode收集各个datanode的报告，当书记块达到最小副本数以上时，会被认为是”安全”的，在一定比例（可设置）的数据块被确定为”安全”后，再过若干时间，安全模式结束。</li>
<li>当检测到副本数不足的数据块是，该快会被复制到达到最小副本数，系统中数据块大的位置并不是由namenode维护的，而是以块列表形式存储在datanode中。</li>
</ul>
<h3 id="Block的副本防置策略"><a href="#Block的副本防置策略" class="headerlink" title="Block的副本防置策略"></a>Block的副本防置策略</h3><ul>
<li>第一个副本：防置在上传的DN；如果是集群外提交，则随机挑选一台磁盘不是太满，CPU不太忙的节点；</li>
<li>第二个副本：防置在于第一个副本不同的机架（Rack）的节点上。</li>
<li>第三个副本：与第二个副本相同机架的节点。</li>
<li>更多副本：随机节点。</li>
</ul>
<h3 id="HDFS写流程"><a href="#HDFS写流程" class="headerlink" title="HDFS写流程"></a>HDFS写流程</h3><p>图</p>
<p>流水线</p>
<p>距离优先</p>
<h3 id="HDFS读流程"><a href="#HDFS读流程" class="headerlink" title="HDFS读流程"></a>HDFS读流程</h3><h3 id="HDFS文件权限-POSIX"><a href="#HDFS文件权限-POSIX" class="headerlink" title="HDFS文件权限 POSIX"></a>HDFS文件权限 POSIX</h3><ul>
<li><p>与Linux文件权限类似</p>
<ul>
<li>r:read; w:write; x:execute</li>
<li>权限x对于文件忽略，对于文件夹表示十分允许访问其内容</li>
</ul>
</li>
<li><p>如果Linux系统用户A使用hadoop命令创建一个文件，那么这个文件在HDFS中的owner就是A；</p>
</li>
<li><p>HDFS的权限目的：组织好人做错事，而不是阻止坏人做坏事。HDFS<br>相信，你告诉我你是谁，我就认为你是谁。</p>
</li>
</ul>
<h2 id="Hadoop2"><a href="#Hadoop2" class="headerlink" title="Hadoop2"></a>Hadoop2</h2><h3 id="Hadoop2产生背景"><a href="#Hadoop2产生背景" class="headerlink" title="Hadoop2产生背景"></a>Hadoop2产生背景</h3><p>Hadoop1中HDFS和MapReduce在高可用、扩展性等方面存在问题</p>
<ul>
<li>NameNode单点故障，难以应用于在线场景， HA</li>
<li>NameNode压力过大，且内存首先，影响扩展性， F<br>MapReduce存在的问题：</li>
<li>JobTracker访问压力大，影响系统扩展性</li>
<li>难以支持出MapReduce之外的计算框架，比如spark、Storm等</li>
</ul>
<p>Hadoop2.x由HDFS、MapReduce和YARN三个分支构成。</p>
<ul>
<li>HDFS： NN Federation（联邦）、HA；2.x只支持2个节点HA<br>，3.0实现一主多从；</li>
<li>MapReduce： 运行在YARN上的MR；离线计算，基于磁盘I/O计算</li>
<li>YARN：资源管理系统</li>
</ul>
<h3 id="HDFS2"><a href="#HDFS2" class="headerlink" title="HDFS2"></a>HDFS2</h3><p>解决HDFS1中单点故障和内存受限的问题</p>
<p>解决单点故障：</p>
<ul>
<li>HDFS HA ：通过贮备NameNode解决</li>
<li>如果中NameNode发生故障，则切换到备NameNode上；</li>
</ul>
<p>解决内存受限问题</p>
<ul>
<li>HDFS Federation 联邦</li>
<li>水平扩展，支持多个NameNode；<ul>
<li>每个NameNode分管一部分目录；</li>
<li>所有NameNode共享所有DataNode存储资源；<br>HDFS2仅是架构上发生了变化，使用方式不变，对HDFS使用者透明，HDFS1中的命令和API仍可以使用。</li>
</ul>
</li>
</ul>
<p>图</p>
<p>Linux NFS<br>JournalNodes<br>最终一致性</p>

        
    

    
</div>


                

                <!-- Post Comments -->
                
                    
                
            </div>

            <!-- Post Prev & Next Nav -->
            <nav class="material-nav mdl-color-text--grey-50 mdl-cell mdl-cell--12-col">
    <!-- Prev Nav -->
    
        <a href="/2019/09/28/Django/" id="post_nav-newer" class="prev-content">
            <button class="mdl-button mdl-js-button mdl-js-ripple-effect mdl-button--icon mdl-color--white mdl-color-text--grey-900" role="presentation">
                <i class="material-icons">arrow_back</i>
            </button>
            &nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;
            新篇
        </a>
    

    <!-- Section Spacer -->
    <div class="section-spacer"></div>

    <!-- Next Nav -->
    
        <a href="/2019/08/04/Redis-Notes/" id="post_nav-older" class="next-content">
            旧篇
            &nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;
            <button class="mdl-button mdl-js-button mdl-js-ripple-effect mdl-button--icon mdl-color--white mdl-color-text--grey-900" role="presentation">
                <i class="material-icons">arrow_forward</i>
            </button>
        </a>
    
</nav>

        </div>
    </div>



                    
                        <!-- Overlay For Active Sidebar -->
<div class="sidebar-overlay"></div>

<!-- Material sidebar -->
<aside id="sidebar" class="sidebar sidebar-colored sidebar-fixed-left" role="navigation">
    <div id="sidebar-main">
        <!-- Sidebar Header -->
        <div class="sidebar-header header-cover" style="background-image: url(/img/sidebar_header.png);">
    <!-- Top bar -->
    <div class="top-bar"></div>

    <!-- Sidebar toggle button -->
    <button type="button" class="sidebar-toggle mdl-button mdl-js-button mdl-js-ripple-effect mdl-button--icon" style="display: initial;" data-upgraded=",MaterialButton,MaterialRipple">
        <i class="material-icons">clear_all</i>
        <span class="mdl-button__ripple-container">
            <span class="mdl-ripple">
            </span>
        </span>
    </button>

    <!-- Sidebar Avatar -->
    <div class="sidebar-image">
        <img src="/img/avatar.png" alt="FWZ's avatar">
    </div>

    <!-- Sidebar Email -->
    <a data-toggle="dropdown" class="sidebar-brand" href="#settings-dropdown">
        youremail@email.com
        <b class="caret"></b>
    </a>
</div>


        <!-- Sidebar Navigation  -->
        <ul class="nav sidebar-nav">
    <!-- User dropdown  -->
    <li class="dropdown">
        <ul id="settings-dropdown" class="dropdown-menu">
            
                <li>
                    <a href="#" target="_blank" title="Email Me">
                        
                            <i class="material-icons sidebar-material-icons sidebar-indent-left1pc-element">email</i>
                        
                        Email Me
                    </a>
                </li>
            
        </ul>
    </li>

    <!-- Homepage -->
    
        <li id="sidebar-first-li">
            <a href="/">
                
                    <i class="material-icons sidebar-material-icons">home</i>
                
                主页
            </a>
        </li>
        
    

    <!-- Archives  -->
    
        <li class="dropdown">
            <a href="#" class="ripple-effect dropdown-toggle" data-toggle="dropdown">
                
                    <i class="material-icons sidebar-material-icons">inbox</i>
                
                    归档
                <b class="caret"></b>
            </a>
            <ul class="dropdown-menu">
            <li>
                <a class="sidebar_archives-link" href="/archives/2019/09/">九月 2019<span class="sidebar_archives-count">1</span></a></li><li><a class="sidebar_archives-link" href="/archives/2019/08/">八月 2019<span class="sidebar_archives-count">3</span></a></li><li><a class="sidebar_archives-link" href="/archives/2019/07/">七月 2019<span class="sidebar_archives-count">7</span></a></li><li><a class="sidebar_archives-link" href="/archives/2019/06/">六月 2019<span class="sidebar_archives-count">3</span></a></li><li><a class="sidebar_archives-link" href="/archives/2019/01/">一月 2019<span class="sidebar_archives-count">1</span></a></li><li><a class="sidebar_archives-link" href="/archives/2018/12/">十二月 2018<span class="sidebar_archives-count">1</span></a></li><li><a class="sidebar_archives-link" href="/archives/2018/07/">七月 2018<span class="sidebar_archives-count">1</span></a></li><li><a class="sidebar_archives-link" href="/archives/2018/06/">六月 2018<span class="sidebar_archives-count">2</span></a></li><li><a class="sidebar_archives-link" href="/archives/2018/05/">五月 2018<span class="sidebar_archives-count">3</span></a>
            </ul>
        </li>
        
    

    <!-- Categories  -->
    

    <!-- Pages  -->
    

    <!-- Article Number  -->
    
</ul>


        <!-- Sidebar Footer -->
        <!--
I'm glad you use this theme, the development is no so easy, I hope you can keep the copyright, I will thank you so much.
If you still want to delete the copyrights, could you still retain the first one? Which namely "Theme Material"
It will not impact the appearance and can give developers a lot of support :)

很高兴您使用并喜欢该主题，开发不易 十分谢谢与希望您可以保留一下版权声明。
如果您仍然想删除的话 能否只保留第一项呢？即 "Theme Material"
它不会影响美观并可以给开发者很大的支持和动力。 :)
-->

<!-- Sidebar Divider -->

    <div class="sidebar-divider"></div>


<!-- Theme Material -->

    <a href="https://github.com/viosey/hexo-theme-material"  class="sidebar-footer-text-a" target="_blank">
        <div class="sidebar-text mdl-button mdl-js-button mdl-js-ripple-effect sidebar-footer-text-div" data-upgraded=",MaterialButton,MaterialRipple">
            主题 - Material
            <span class="sidebar-badge badge-circle">i</span>
        </div>
    </a>


<!-- Help & Support -->
<!--

-->

<!-- Feedback -->
<!--

-->

<!-- About Theme -->
<!--

-->

    </div>

    <!-- Sidebar Image -->
    

</aside>

                    

                    
                        <!-- Footer Top Button -->
                        <div id="back-to-top" class="toTop-wrap">
    <a href="#top" class="toTop">
        <i class="material-icons footer_top-i">expand_less</i>
    </a>
</div>

                    

                    <!--Footer-->
<footer class="mdl-mini-footer" id="bottom">
    
    <!-- Paradox Footer Left Section -->
    <div class="mdl-mini-footer--left-section sns-list">
    <!-- Twitter -->
    
        <a href="https://twitter.com/twitter" target="_blank">
            <button class="mdl-mini-footer--social-btn social-btn footer-sns-twitter">
                <span class="visuallyhidden">Twitter</span>
            </button><!--
     --></a>
    

    <!-- Facebook -->
    
        <a href="https://www.facebook.com/facebook" target="_blank">
            <button class="mdl-mini-footer--social-btn social-btn footer-sns-facebook">
                <span class="visuallyhidden">Facebook</span>
            </button><!--
     --></a>
    

    <!-- Google + -->
    
        <a href="https://www.google.com/" target="_blank">
            <button class="mdl-mini-footer--social-btn social-btn footer-sns-gplus">
                <span class="visuallyhidden">Google Plus</span>
            </button><!--
     --></a>
    

    <!-- Weibo -->
    

    <!-- Instagram -->
    

    <!-- Tumblr -->
    

    <!-- Github -->
    

    <!-- LinkedIn -->
    

    <!-- Zhihu -->
    

    <!-- Bilibili -->
    

    <!-- Telegram -->
    
    
    <!-- V2EX -->
    
</div>


    <!--Copyright-->
    <div id="copyright">
        Copyright&nbsp;©
        <script
            type="text/javascript">var fd = new Date(); document.write("&nbsp;" + fd.getFullYear() + "&nbsp;");</script>
        fengweizan's BLOG
        
    </div>

    <!-- Paradox Footer Right Section -->

    <!--
        I am glad you use this theme, the development is no so easy, I hope you can keep the copyright.
        It will not impact the appearance and can give developers a lot of support :)

        很高兴您使用该主题，开发不易，希望您可以保留一下版权声明。
        它不会影响美观并可以给开发者很大的支持。 :)
        -->

    <div class="mdl-mini-footer--right-section">
        <div>
            <div class="footer-develop-div">Powered by <a href="https://hexo.io" target="_blank"
                    class="footer-develop-a">Hexo</a></div>
            <div class="footer-develop-div">Theme - <a href="https://github.com/viosey/hexo-theme-material"
                    target="_blank" class="footer-develop-a">Material</a></div>
        </div>
    </div>
    
</footer>



                    <!-- Import JS File -->

    <script>lsloader.load("lazyload_js","/js/lazyload.min.js?1BcfzuNXqV+ntF6gq+5X3Q==", true)</script>



    <script>lsloader.load("js_js","/js/js.min.js?V/53wGualMuiPM3xoetD5Q==", true)</script>



    <script>lsloader.load("np_js","/js/nprogress.js?pl3Qhb9lvqR1FlyLUna1Yw==", true)</script>


<script type="text/ls-javascript" id="NProgress-script">
    NProgress.configure({
        showSpinner: true
    });
    NProgress.start();
    $('#nprogress .bar').css({
        'background': '#29d'
    });
    $('#nprogress .peg').css({
        'box-shadow': '0 0 10px #29d, 0 0 15px #29d'
    });
    $('#nprogress .spinner-icon').css({
        'border-top-color': '#29d',
        'border-left-color': '#29d'
    });
    setTimeout(function() {
        NProgress.done();
        $('.fade').removeClass('out');
    }, 800);
</script>













<!-- UC Browser Compatible -->
<script>
	var agent = navigator.userAgent.toLowerCase();
	if(agent.indexOf('ucbrowser')>0) {
		document.write('<link rel="stylesheet" href="/css/uc.css">');
	   alert('由于 UC 浏览器使用极旧的内核，而本网站使用了一些新的特性。\n为了您能更好的浏览，推荐使用 Chrome 或 Firefox 浏览器。');
	}
</script>

<!-- Import prettify js  -->

    
        
            <script>lsloader.load("prettify_js","/js/prettify.min.js?WN07fivHQSMKWy7BmHBB6w==", true)</script>
        
    



<!-- Window Load -->
<!-- add class for prettify -->
<script type="text/ls-javascript" id="window-load">
    $(window).on('load', function() {
        // Post_Toc parent position fixed
        $('.post-toc-wrap').parent('.mdl-menu__container').css('position', 'fixed');
    });

    
        
            $(function() {
                $('pre').addClass('prettyprint linenums').attr('style', 'overflow:auto;');
                prettyPrint();
                })
        
    
    
</script>

<!-- MathJax Load-->


<!-- Bing Background -->

<script type="text/ls-javascript" id="Bing-Background-script">
    queue.offer(function(){
        $('body').attr('data-original', 'https://api.i-meto.com/bing?');
    });
</script>


<script type="text/ls-javascript" id="lazy-load">
    // Offer LazyLoad
    queue.offer(function(){
        $('.lazy').lazyload({
            effect : 'show'
        });
    });

    // Start Queue
    $(document).ready(function(){
        setInterval(function(){
            queue.execNext();
        },200);
    });
</script>

<!-- Custom Footer -->



<script>
    (function(){
        var scriptList = document.querySelectorAll('script[type="text/ls-javascript"]')

        for (var i = 0; i < scriptList.length; ++i) {
            var item = scriptList[i];
            lsloader.runInlineScript(item.id,item.id);
        }
    })()
console.log('\n %c © Material Theme | Version: 1.5.2 | https://github.com/viosey/hexo-theme-material %c \n', 'color:#455a64;background:#e0e0e0;padding:5px 0;border-top-left-radius:5px;border-bottom-left-radius:5px;', 'color:#455a64;background:#e0e0e0;padding:5px 0;border-top-right-radius:5px;border-bottom-right-radius:5px;');
</script>

                </main>
            </div><!-- hexo-inject:begin --><!-- hexo-inject:end -->
        </body>
    
</html>
